package find.prj.prog;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HTMLPravdaNewsReaderConsole {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// Reading main page of pravda.com.ua and saving all news URLs in urlStrings
		String mainPageString = null;
		try {
			mainPageString = readContentFromURLToString("http://www.epravda.com.ua/news/2013/05/2/373712/");
		} catch (IOException e) {
			e.printStackTrace();
		}
//		ArrayList<String> urlStrings = getAllPravdaNewsURLs (mainPageString);
			
		Article article = Article.readFromPravdaNewsHTMLString(mainPageString);
//		System.out.println(article.toString());
				
/*		for (String url : urlStrings) {
			String newsContent="";
			try {
				 newsContent = readContentFromURLToString(url);
			} catch (IOException e) {
				e.printStackTrace();
			}
			Article article = Article.readFromPravdaNewsHTMLString(newsContent);
			if (article != null)
				System.out.println(article.toString() + "\n\n**************************************************************\n");
		}*/
	}

	/**
	 * Reading content from URL and returning it as a String
	 * 
	 * @param urlString - URL string
	 * @return
	 * @throws IOException - thrown when can't open a HttpURLConnection
	 */
	private static String readContentFromURLToString (String urlString) throws IOException{
		HttpURLConnection httpCon = null;
		URL url = new URL(urlString);
		StringBuilder content = new StringBuilder();

		httpCon = (HttpURLConnection)url.openConnection();
		httpCon.setRequestMethod("GET");
	    BufferedReader br = new BufferedReader(new InputStreamReader((InputStream) httpCon.getContent()));
	    String line = br.readLine();
	    while (line != null) {
	      content.append(line + "\n");
	      line = br.readLine();
	    }
	    httpCon.disconnect(); 

		return content.replace(content.length()-1, content.length(), "").toString(); 
	}
	
	private static ArrayList<String> getAllPravdaNewsURLs (String content) {

		// Grouping all <dl class="mnews1" ... </dl> items in newsArticlesString
		Pattern pattern = Pattern.compile("\\<dl class=\"mnews1\".+?\\</dl\\>", Pattern.DOTALL | Pattern.MULTILINE);
		Matcher matcher = pattern.matcher(content);
		String newsBlocks="";
		while (matcher.find())
			newsBlocks += content.substring(matcher.start(), matcher.end());

		// Searching for links in newsBlocks.
		// If link is relative - prefix it with http://www.pravda.com.ua . Otherwise - keep unchanged.
		// Save all links into urlStrings
		pattern = Pattern.compile("\\<a href=.+?[ >]", Pattern.DOTALL | Pattern.MULTILINE);
		matcher = pattern.matcher(newsBlocks);
		ArrayList<String> urlStrings= new ArrayList<String>();
		String url;
		while (matcher.find()) {
			url = newsBlocks.substring(matcher.start(), matcher.end()).trim().replaceAll("<a href=", "").replaceAll(">", "");
			if (url.startsWith("/")) 
				url = "http://pravda.com.ua/news" + url;
			urlStrings.add(url);
		}
		return urlStrings;
		
	}

	
	/**
	 * Nested class for saving www.pravda.com.ua/news/ article data.
	 * No public constructor. To create instance of Article use one of read methods
	 *  
	 * @author gylka
	 *
	 */
	private static class Article {
		
		private String Title;
		private String FullText;
		
		// No public constructor. To create instance of Article use one of read methods  
		private Article() { };
		
		/**
		 * Method reads article from HTML in String by searching for <div class="text" and its close tag, 
		 * removing all tags inside it and putting it in Article.Title field. Same with title - <div class="title". 
		 * 
		 * @param newsHTMLString - HTML loaded from www.pravda.com.ua/news/ and converted to String
		 * 
		 * @return - Article object
		 */
		public static Article readFromPravdaNewsHTMLString (String newsHTMLString) {
			Article article = new Article();
			
			// Searching for <div class="text" and closing with first </div> making substring of it to fulltext String
			Pattern pattern = Pattern.compile("\\<div class=\"text\".+?\\</div\\>", Pattern.DOTALL | Pattern.MULTILINE);
			Matcher matcher = pattern.matcher(newsHTMLString);
			String fulltext = "";
			if (matcher.find())
				fulltext = newsHTMLString.substring(matcher.start(), matcher.end());
				
			// Removing scripts
			pattern = Pattern.compile("\\<script.+?/script\\>", Pattern.DOTALL | Pattern.MULTILINE);
			matcher = pattern.matcher(fulltext);
			ArrayList<String> ss = new ArrayList<String>();
			String s;
			while (matcher.find()) {
				s = fulltext.substring(matcher.start(), matcher.end());
				System.out.println(s);
				ss.add(fulltext.substring(matcher.start(), matcher.end()));
			}

			// Removing <div class="tags" from fulltext String.
			// Replacing &nbsp; with " "
			// Replacing &quot; with \"
			// Replacing &rsquo; with \'
			fulltext = fulltext.replaceAll("\\<script.+?/script\\>", "").replaceAll("\\<p class=\"tags\".+?\\</p\\>", "").replaceAll("&nbsp;"," ").replaceAll("&quot;", "\"").replaceAll("&rsquo;","\'");

			// Replacing all tags
			// Replacing 2+ line-breaks with/without spaces with 2 line-breaks			
			fulltext = fulltext.replaceAll("\\<.+?\\>", "").replaceAll("(\n *){2,}", "\n\n").trim();
			article.FullText = fulltext;

			// Searching for <h1 class="title" and closing with first </h1> making substring of it to title String
			pattern = Pattern.compile("\\<h1 class=\"title\".+?\\</h1\\>", Pattern.DOTALL | Pattern.MULTILINE);
			matcher = pattern.matcher(newsHTMLString);
			String title = "";
			if (matcher.find())
				title = newsHTMLString.substring(matcher.start(), matcher.end());
				
			// Replacing &nbsp with " "
			// Replacing &rsquo; with \'
			// Replacing &quot; with \"
			// Replacing all tags
			title = title.replaceAll("&nbsp;"," ").replaceAll("&quot;", "\"").replaceAll("&rsquo;","\'").replaceAll("<.+?>", "").trim();
			article.Title = title;
			
			return article;
		}
		
		public String toString() {
			return (new String() + this.Title + "\n\n" + this.FullText); 
		}
		
		
	}
	
}
